import requests
from lxml import etree
import time
import datetime
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
# es = Elasticsearch(['127.0.0.1:9200'])
es = Elasticsearch(['219.224.134.226:9202'])
# 上证e互动
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.80 Safari/537.36'
}

t = {
    '最新答复': 11,
    '最新提问': 10,
    '热门': 12,
    '热推': 13,
}


def get_result(index,doc_type,mode=1,sdate=None,edate=None):
    """mode
        1 表示抓取最新回复
        2 表示按照时间段抓取
    """
    page = 1
    while 1:
        if mode == 1:
            url = 'http://sns.sseinfo.com/ajax/feeds.do?page={}&type=11&pageSize=500&lastid=-1&show=1&_={}'.format(
                page,int(round(time.time() * 1000)))
            r = requests.get(url=url, headers=headers, timeout=5)
            # print(r.text)
        if mode == 2:
            if not sdate or not edate:
                raise ValueError('if mode==2,please check sdate and edate!!')
            url = 'http://sns.sseinfo.com/getNewData.do'
            data = {
                "sdate": sdate,
                "edate": edate,
                "keyword": "",
                "type": "1",
                "page": "{}".format(page),
                "comId": ""
            }
            print(sdate,page)
            r = requests.post(url=url, headers=headers, data=data, timeout=5)
        r.encoding='utf-8'
        tree = etree.HTML(r.text)
        m_feed_items = tree.xpath('//div[@class="m_feed_item"]')
        result = []
        for item in m_feed_items:
            dic = {}
            m_qa_detail = item.xpath('./div[@class="m_feed_detail m_qa_detail"]')
            m_qa = item.xpath('./div[@class="m_feed_detail m_qa"]')
            if not m_qa:
                continue
            m_qa_detail = m_qa_detail[0]
            m_qa = m_qa[0]
            dic['id'] = item.xpath('./@id')[0].lstrip('item-')
            dic['ask_name'] = m_qa_detail.xpath('./div[@class="m_feed_face"]/a/@title')[0]
            try:
                dic['company'] = m_qa_detail.xpath('./div[@class="m_feed_cnt "]/div[@class="m_feed_txt"]/a/text()')[0].lstrip(':').lstrip('@').split('(')[0]
            except:
                continue
            dic['ask_text'] = ''.join(m_qa_detail.xpath(
                './div[@class="m_feed_cnt "]/div[@class="m_feed_txt"]/text()')).strip()
            if mode==1:
                ask_time = m_qa_detail.xpath('./div[contains(@class,"m_feed_cnt")]/div[@class="m_feed_func"]/div[@class="m_feed_from"]/span/text()')[0]
                dic['ask_ts'], dic['ask_date'] = trans_date_ts(ask_time)
            dic['answer_text'] = m_qa.xpath('./div[@class="m_feed_cnt"]/div[@class="m_feed_txt"]/text()')[0].strip()
            answer_time = m_qa.xpath('./div[@class="m_feed_func top10"]/div[@class="m_feed_from"]/span/text()')[0]
            if mode == 2:
                answer_time = sdate[:4]+ '年' + answer_time
            dic['answer_ts'], dic['answer_date'] = trans_date_ts(answer_time)
            result.append({
                '_index': index,
                '_type': doc_type,
                '_id': dic['id'],
                '_source': dic})
            # print(dic)
        if result:
            bulk(es,result)
        page += 1
        end = tree.xpath('//a[@class="m_feed_note"]/text()')
        if end and end[0] == '暂时没有问答内容':
            break
        time.sleep(2)


def trans_date_ts(date):
    # 转成时间戳
    date = date.replace('年', '-').replace('月', '-').replace('日', '')
    now_ts = time.time()
    if '分钟前' in date:
        minute = int(date.rstrip('分钟前'))
        ts = now_ts - minute * 60
        date = ts2date(ts)
    elif '小时前' in date:
        hour = int(date.rstrip('小时前'))
        ts = now_ts - hour * 3600
        date = ts2date(ts)
    elif '昨天' in date:
        new_time = date.lstrip('昨天')
        date = get_before_date(1) + new_time + ":00"
    elif '今天' in date:
        new_time = date.lstrip('今天')
        date = get_before_date(0) + new_time + ":00"
    elif '刚刚' in date:
        date = ts2date(time.time())
    elif '秒前' in date:
        sec = int(date.rstrip('秒前'))
        ts = now_ts - sec
        date = ts2date(ts)
    elif date.count('-') == 1:
        date = time.strftime("%Y-", time.localtime()) + date
    # print(date)
    return date2ts(date), date[:10]


def date2ts(date):
    try:
        return int(time.mktime(time.strptime(date, '%Y-%m-%d')))
    except:
        try:
            return int(time.mktime(time.strptime(date, '%Y-%m-%d %H:%M')))
        except:
            return int(time.mktime(time.strptime(date, '%Y-%m-%d %H:%M:%S')))


def ts2date(ts):
    return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))


def get_before_date(date_cha,date=None):
    # date_cha为当天日期的前几天,返回前几天的日期
    # 如当天2018-7-7,date_cha=1,则返回2018-7-6
    if not date:
        today = datetime.date.today()
    else:
        today = datetime.datetime.strptime(date, '%Y-%m-%d')
    oneday = datetime.timedelta(days=date_cha)
    newday = today - oneday
    return str(newday)[:10]
def get_today():
    date = time.strftime('%Y-%m-%d',time.localtime())
    return date
def main():
    ### mode==1时 返回结果有发问时间
    ### mode==2是,返回结果没有发问时间
    index = 'info'
    doc_type='see'
    # 根据默认的最新答复链接抓取并存入es,直到返回结果为空停止
    # get_result(mode=1,index=index,doc_type=doc_type)
    # 根据传入的答复日期抓取并存入es
    today = get_today()
    date = '2014-02-28'
    while date<today:
        get_result(mode=2,sdate=date,edate=date,index=index,doc_type=doc_type)
        date = get_before_date(-1,date)
        # time.sleep(30)

if __name__ == '__main__':
    main()
    # trans_date_ts('2019-02-29 08:34')
